{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-698ada706af1>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一个非常非常简陋的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model\n",
    "h1_rank=300\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "\n",
    "#W1 = tf.Variable(tf.zeros([784, h1_rank]))\n",
    "W1 = tf.Variable(tf.truncated_normal([784, h1_rank],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([h1_rank]))\n",
    "\n",
    "hidden1 = tf.nn.relu(tf.matmul(x,W1)+b1)\n",
    "#hidden1 = tf.nn.swish(tf.matmul(x,W1)+b1)\n",
    "#hidden1 = tf.nn.sigmoid(tf.matmul(x,W1)+b1)\n",
    "hidden1_drop = tf.nn.dropout(hidden1,keep_prob)\n",
    "\n",
    "W2 = tf.Variable(tf.zeros([h1_rank,10]))\n",
    "b2 = tf.Variable(tf.zeros([10]))\n",
    "y = tf.matmul(hidden1_drop, W2) + b2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义我们的ground truth 占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-5-44d2ae255ef4>:10: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_step = tf.train.GradientDescentOptimizer(0.25).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "然后我们运行12k个step(20 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train\n",
    "for _ in range(12000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys,keep_prob:0.75})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9825\n"
     ]
    }
   ],
   "source": [
    "  # Test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels,keep_prob:1.0}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 0.0004961 ,  0.02741734, -0.03624421, ..., -0.14412028,\n",
       "          0.19727139,  0.0433793 ],\n",
       "        [-0.05994942,  0.02787312,  0.11099777, ..., -0.02337455,\n",
       "          0.05930616,  0.03557725],\n",
       "        [ 0.1286029 , -0.12328122,  0.10585215, ...,  0.01555003,\n",
       "         -0.00155922,  0.01445904],\n",
       "        ...,\n",
       "        [ 0.03562078, -0.03003085, -0.0136334 , ..., -0.10143226,\n",
       "          0.01641271,  0.00274235],\n",
       "        [-0.0591175 , -0.05207638, -0.06546157, ..., -0.0686275 ,\n",
       "          0.19154376, -0.17631231],\n",
       "        [ 0.07329471, -0.12693317, -0.0185167 , ..., -0.0116551 ,\n",
       "         -0.12991796,  0.09698704]], dtype=float32),\n",
       " array([ 6.47338405e-02,  6.13677390e-02, -2.37604044e-02, -9.82131585e-02,\n",
       "         6.04792424e-02, -7.33092204e-02, -1.35229543e-01,  9.69380736e-02,\n",
       "        -6.86375285e-03,  5.24679199e-02, -1.44736478e-02, -1.30800292e-01,\n",
       "        -3.55424248e-02,  7.59277493e-02, -2.52257586e-02, -1.38956770e-01,\n",
       "         4.38091792e-02,  6.07176796e-02,  6.14937767e-02, -4.60011363e-02,\n",
       "        -4.08599190e-02, -2.36046501e-02, -8.41590315e-02, -2.36734170e-02,\n",
       "        -9.49134082e-02,  3.06037045e-03,  1.55408978e-02, -6.08876012e-02,\n",
       "        -3.95240448e-02, -3.36144120e-02, -5.01791248e-03, -1.38976201e-01,\n",
       "        -1.99716222e-02, -1.19943380e-01,  4.25542518e-02,  7.85734057e-02,\n",
       "        -6.59478158e-02,  1.54204637e-01,  2.07509007e-02,  3.89243774e-02,\n",
       "         3.85664627e-02, -1.25977974e-02,  2.68589873e-02,  9.12753195e-02,\n",
       "         1.13675389e-02,  5.02232499e-02,  8.89674798e-02, -1.12189027e-02,\n",
       "         2.15793803e-01, -1.11273490e-01, -7.11674094e-02,  1.08469343e-02,\n",
       "         7.56025733e-03,  4.53087278e-02, -3.72122750e-02,  4.81047519e-02,\n",
       "        -4.25809957e-02, -3.54740880e-02,  2.80188061e-02, -4.20887358e-02,\n",
       "        -2.42173299e-02,  4.59729172e-02,  8.20254609e-02,  8.83235335e-02,\n",
       "         2.83367597e-02, -1.89011507e-02, -7.83280283e-02, -7.68712163e-02,\n",
       "         4.33146302e-03, -3.69930752e-02, -6.05866052e-02,  9.22440365e-02,\n",
       "         5.55674583e-02, -1.47564009e-01,  8.70303884e-02,  4.64907102e-02,\n",
       "         6.64074197e-02,  6.32842556e-02, -8.76692235e-02, -8.82288162e-03,\n",
       "         5.10667302e-02,  1.45771503e-01,  2.88347397e-02,  7.75172040e-02,\n",
       "         1.12891190e-01,  3.59449126e-02,  1.23805687e-01,  3.03574968e-02,\n",
       "         9.12931561e-03, -1.28902458e-02, -5.05247526e-02,  2.40363926e-02,\n",
       "         1.09530780e-02,  4.74396199e-02,  5.58363721e-02,  1.23796212e-02,\n",
       "        -5.81609197e-02, -2.92361435e-02, -1.40127778e-01, -7.64677897e-02,\n",
       "        -2.09665224e-02, -2.93193152e-05,  9.45682377e-02,  2.79974397e-02,\n",
       "         1.67671293e-02,  3.51155549e-02, -2.62453128e-02,  5.79974391e-02,\n",
       "        -2.19347253e-02,  1.50018513e-01, -7.42199505e-03,  4.01636139e-02,\n",
       "         1.13274135e-01, -9.07395221e-03, -2.45404448e-02,  6.42540157e-02,\n",
       "        -4.50148800e-04,  1.22177072e-01, -9.34403092e-02,  7.73436055e-02,\n",
       "        -1.07024223e-01,  1.75880507e-01, -2.78908741e-02, -1.75961549e-03,\n",
       "        -5.57277799e-02, -7.81273171e-02,  7.25892335e-02,  2.49897875e-02,\n",
       "         3.66534339e-03, -7.44835287e-02,  1.48283139e-01, -4.01170626e-02,\n",
       "        -1.33080751e-01, -4.77938317e-02,  2.15690255e-01,  2.04731431e-02,\n",
       "         1.29572406e-01,  4.53024209e-02, -1.04615867e-01, -5.09192683e-02,\n",
       "         2.02864017e-02,  1.16741601e-02, -1.74652278e-01,  1.69973001e-01,\n",
       "        -2.33364105e-02, -5.51235154e-02, -6.81195259e-02, -4.57231849e-02,\n",
       "         5.25354892e-02, -1.55903891e-01, -5.67175150e-02, -1.07989430e-01,\n",
       "        -6.79326206e-02, -7.24689588e-02,  5.36751933e-02,  1.18127717e-02,\n",
       "         9.70573351e-02,  4.97819819e-02,  1.63446032e-02,  4.75245789e-02,\n",
       "         9.31289606e-03,  5.18768877e-02,  6.47689495e-03, -2.39997655e-02,\n",
       "        -6.50115237e-02, -6.87086359e-02, -2.63426341e-02, -5.14880642e-02,\n",
       "         2.67794309e-03, -4.70288768e-02, -8.39830711e-02,  1.09764799e-01,\n",
       "         8.30905512e-02, -4.71836366e-02,  1.16855036e-02,  3.38361189e-02,\n",
       "        -4.39862870e-02, -1.24094225e-01, -1.29595154e-03,  7.06725493e-02,\n",
       "        -4.98998240e-02,  3.98018248e-02,  3.09701115e-02, -7.05320239e-02,\n",
       "        -1.26579270e-01, -1.61867246e-01,  1.00645460e-01,  7.05918744e-02,\n",
       "         1.30755663e-01,  7.37196580e-02, -8.60154405e-02, -1.01058312e-01,\n",
       "        -3.11885662e-02, -1.83356907e-02, -1.37146395e-02,  2.59594675e-02,\n",
       "        -1.13541625e-01, -7.55512342e-02,  5.94343338e-03,  1.44240158e-02,\n",
       "         2.81147771e-02, -1.06919073e-02,  9.38301235e-02, -5.93399219e-02,\n",
       "        -3.99990194e-02, -1.16936550e-01, -8.99596736e-02, -6.85553178e-02,\n",
       "        -5.01597896e-02, -8.40030462e-02,  1.16913550e-01, -2.21221223e-02,\n",
       "        -3.54102924e-02, -3.09306867e-02,  9.83802378e-02, -2.84472592e-02,\n",
       "         3.31764296e-02, -2.04744004e-02, -1.11983217e-01,  1.17797606e-01,\n",
       "        -1.17581286e-01,  1.70464888e-02,  7.06532970e-02,  2.87961531e-02,\n",
       "        -4.60046604e-02,  2.41751522e-02,  8.09614956e-02,  1.02744117e-01,\n",
       "        -8.93512741e-03,  1.75891295e-01, -1.00987278e-01, -8.84130038e-03,\n",
       "        -2.33979709e-02,  1.64587162e-02, -2.04976305e-01, -3.62329893e-02,\n",
       "         4.45190184e-02,  5.19402847e-02, -6.44281954e-02,  3.35112065e-02,\n",
       "        -2.05859318e-02,  4.37737368e-02, -1.14566432e-02, -7.47129768e-02,\n",
       "         2.19106898e-01,  1.37128290e-02, -2.20838399e-03,  3.91989090e-02,\n",
       "        -2.97883376e-02, -2.34996192e-02, -1.28136560e-01, -1.20166294e-01,\n",
       "         7.72703215e-02,  3.35565209e-02, -4.70025912e-02, -2.71955729e-02,\n",
       "         8.05194452e-02, -1.74765453e-01,  1.24949031e-01, -7.63974637e-02,\n",
       "         8.15082621e-03, -1.07765973e-01, -1.68412942e-02, -1.21103436e-01,\n",
       "        -7.35629629e-03, -9.11864787e-02, -9.07099396e-02, -2.51219179e-02,\n",
       "        -8.48561153e-02, -4.90614921e-02,  2.05148607e-02, -7.27521107e-02,\n",
       "         6.54781461e-02, -2.94372831e-02, -4.82504815e-02, -1.19096443e-01,\n",
       "         1.53126875e-02, -3.74645628e-02,  5.93702607e-02, -9.48896259e-02,\n",
       "         2.19450481e-02,  1.19913273e-01,  1.99629478e-02,  4.00873721e-02,\n",
       "        -4.24230248e-02,  4.64229546e-02,  2.24943057e-01, -4.10058685e-02,\n",
       "        -2.29730867e-02, -2.67731454e-02, -2.94251651e-01,  6.59430772e-02,\n",
       "         1.89109221e-02,  2.75959130e-02,  4.22556605e-03,  9.94600914e-03,\n",
       "         3.87162194e-02, -1.82972088e-01,  3.54822725e-02,  1.85115457e-01],\n",
       "       dtype=float32),\n",
       " array([[-0.50352997, -0.49989623, -0.23162724, ...,  0.16563092,\n",
       "          0.42297816,  0.39951655],\n",
       "        [-0.18157242,  0.11902218, -0.9033215 , ...,  0.26484206,\n",
       "         -0.02536519,  0.27021933],\n",
       "        [ 0.16845793,  0.06058787,  0.4883201 , ...,  0.39073262,\n",
       "         -0.10837592, -0.35761636],\n",
       "        ...,\n",
       "        [ 0.23820002, -0.09300721,  0.25128835, ...,  0.17878279,\n",
       "          0.1624125 ,  0.30963174],\n",
       "        [-0.05859819,  0.3657486 , -0.14391467, ...,  0.15272985,\n",
       "         -0.6008264 ,  0.30696124],\n",
       "        [ 0.262221  ,  0.24215765,  0.24156874, ...,  0.23394458,\n",
       "         -0.4970117 , -0.9235496 ]], dtype=float32),\n",
       " array([-0.03112213,  0.01551676, -0.23188776, -0.19744432,  0.09266186,\n",
       "        -0.01498853, -0.00535101, -0.23937228,  0.44021943,  0.17175204],\n",
       "       dtype=float32)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([W1,b1,W2,b2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
